{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Transformer-Torch",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "language": "python",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "pycharm": {
      "stem_cell": {
        "cell_type": "raw",
        "source": [],
        "metadata": {
          "collapsed": false
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "g831xANXh2HY"
      },
      "source": [
        "'''\n",
        "  code by Tae Hwan Jung(Jeff Jung) @graykode, Derek Miller @dmmiller612, modify by wmathor\n",
        "  Reference : https://github.com/jadore801120/attention-is-all-you-need-pytorch\n",
        "              https://github.com/JayParks/transformer\n",
        "'''\n",
        "import math\n",
        "import torch\n",
        "import numpy as np\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.utils.data as Data\n",
        "\n",
        "# S: Symbol that shows starting of decoding input\n",
        "# E: Symbol that shows starting of decoding output\n",
        "# P: Symbol that will fill in blank sequence if current batch data size is short than time steps\n",
        "sentences = [\n",
        "        # enc_input           dec_input         dec_output\n",
        "        ['ich mochte ein bier P', 'S i want a beer .', 'i want a beer . E'],\n",
        "        ['ich mochte ein cola P', 'S i want a coke .', 'i want a coke . E']\n",
        "]\n",
        "\n",
        "# Padding Should be Zero\n",
        "src_vocab = {'P' : 0, 'ich' : 1, 'mochte' : 2, 'ein' : 3, 'bier' : 4, 'cola' : 5}\n",
        "src_vocab_size = len(src_vocab)\n",
        "\n",
        "tgt_vocab = {'P' : 0, 'i' : 1, 'want' : 2, 'a' : 3, 'beer' : 4, 'coke' : 5, 'S' : 6, 'E' : 7, '.' : 8}\n",
        "idx2word = {i: w for i, w in enumerate(tgt_vocab)}\n",
        "tgt_vocab_size = len(tgt_vocab)\n",
        "\n",
        "src_len = 5 # enc_input max sequence length\n",
        "tgt_len = 6 # dec_input(=dec_output) max sequence length\n",
        "\n",
        "# Transformer Parameters\n",
        "d_model = 512  # Embedding Size\n",
        "d_ff = 2048 # FeedForward dimension\n",
        "d_k = d_v = 64  # dimension of K(=Q), V\n",
        "n_layers = 6  # number of Encoder of Decoder Layer\n",
        "n_heads = 8  # number of heads in Multi-Head Attention"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m6PIfZqsBnMK"
      },
      "source": [
        "def make_data(sentences):\n",
        "    enc_inputs, dec_inputs, dec_outputs = [], [], []\n",
        "    for i in range(len(sentences)):\n",
        "      enc_input = [[src_vocab[n] for n in sentences[i][0].split()]] # [[1, 2, 3, 4, 0], [1, 2, 3, 5, 0]]\n",
        "      dec_input = [[tgt_vocab[n] for n in sentences[i][1].split()]] # [[6, 1, 2, 3, 4, 8], [6, 1, 2, 3, 5, 8]]\n",
        "      dec_output = [[tgt_vocab[n] for n in sentences[i][2].split()]] # [[1, 2, 3, 4, 8, 7], [1, 2, 3, 5, 8, 7]]\n",
        "\n",
        "      enc_inputs.extend(enc_input)\n",
        "      dec_inputs.extend(dec_input)\n",
        "      dec_outputs.extend(dec_output)\n",
        "\n",
        "    return torch.LongTensor(enc_inputs), torch.LongTensor(dec_inputs), torch.LongTensor(dec_outputs)\n",
        "\n",
        "enc_inputs, dec_inputs, dec_outputs = make_data(sentences)\n",
        "\n",
        "class MyDataSet(Data.Dataset):\n",
        "  def __init__(self, enc_inputs, dec_inputs, dec_outputs):\n",
        "    super(MyDataSet, self).__init__()\n",
        "    self.enc_inputs = enc_inputs\n",
        "    self.dec_inputs = dec_inputs\n",
        "    self.dec_outputs = dec_outputs\n",
        "  \n",
        "  def __len__(self):\n",
        "    return self.enc_inputs.shape[0]\n",
        "  \n",
        "  def __getitem__(self, idx):\n",
        "    return self.enc_inputs[idx], self.dec_inputs[idx], self.dec_outputs[idx]\n",
        "\n",
        "loader = Data.DataLoader(MyDataSet(enc_inputs, dec_inputs, dec_outputs), 2, True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YN_OYtMjPnRg"
      },
      "source": [
        "class PositionalEncoding(nn.Module):\n",
        "    def __init__(self, d_model, dropout=0.1, max_len=5000):\n",
        "        super(PositionalEncoding, self).__init__()\n",
        "        self.dropout = nn.Dropout(p=dropout)\n",
        "\n",
        "        pe = torch.zeros(max_len, d_model)\n",
        "        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)\n",
        "        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))\n",
        "        pe[:, 0::2] = torch.sin(position * div_term)\n",
        "        pe[:, 1::2] = torch.cos(position * div_term)\n",
        "        pe = pe.unsqueeze(0).transpose(0, 1)\n",
        "        self.register_buffer('pe', pe)\n",
        "\n",
        "    def forward(self, x):\n",
        "        '''\n",
        "        x: [seq_len, batch_size, d_model]\n",
        "        '''\n",
        "        x = x + self.pe[:x.size(0), :]\n",
        "        return self.dropout(x)\n",
        "\n",
        "def get_attn_pad_mask(seq_q, seq_k):\n",
        "    '''\n",
        "    seq_q: [batch_size, seq_len]\n",
        "    seq_k: [batch_size, seq_len]\n",
        "    seq_len could be src_len or it could be tgt_len\n",
        "    seq_len in seq_q and seq_len in seq_k maybe not equal\n",
        "    '''\n",
        "    batch_size, len_q = seq_q.size()\n",
        "    batch_size, len_k = seq_k.size()\n",
        "    # eq(zero) is PAD token\n",
        "    pad_attn_mask = seq_k.data.eq(0).unsqueeze(1)  # [batch_size, 1, len_k], False is masked\n",
        "    return pad_attn_mask.expand(batch_size, len_q, len_k)  # [batch_size, len_q, len_k]\n",
        "\n",
        "def get_attn_subsequence_mask(seq):\n",
        "    '''\n",
        "    seq: [batch_size, tgt_len]\n",
        "    '''\n",
        "    attn_shape = [seq.size(0), seq.size(1), seq.size(1)]\n",
        "    subsequence_mask = np.triu(np.ones(attn_shape), k=1) # Upper triangular matrix\n",
        "    subsequence_mask = torch.from_numpy(subsequence_mask).byte()\n",
        "    return subsequence_mask # [batch_size, tgt_len, tgt_len]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lrXEC4AogFfG"
      },
      "source": [
        "class ScaledDotProductAttention(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(ScaledDotProductAttention, self).__init__()\n",
        "\n",
        "    def forward(self, Q, K, V, attn_mask):\n",
        "        '''\n",
        "        Q: [batch_size, n_heads, len_q, d_k]\n",
        "        K: [batch_size, n_heads, len_k, d_k]\n",
        "        V: [batch_size, n_heads, len_v(=len_k), d_v]\n",
        "        attn_mask: [batch_size, n_heads, seq_len, seq_len]\n",
        "        '''\n",
        "        scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k) # scores : [batch_size, n_heads, len_q, len_k]\n",
        "        scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is True.\n",
        "        \n",
        "        attn = nn.Softmax(dim=-1)(scores)\n",
        "        context = torch.matmul(attn, V) # [batch_size, n_heads, len_q, d_v]\n",
        "        return context, attn\n",
        "\n",
        "class MultiHeadAttention(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(MultiHeadAttention, self).__init__()\n",
        "        self.W_Q = nn.Linear(d_model, d_k * n_heads, bias=False)\n",
        "        self.W_K = nn.Linear(d_model, d_k * n_heads, bias=False)\n",
        "        self.W_V = nn.Linear(d_model, d_v * n_heads, bias=False)\n",
        "        self.fc = nn.Linear(n_heads * d_v, d_model, bias=False)\n",
        "    def forward(self, input_Q, input_K, input_V, attn_mask):\n",
        "        '''\n",
        "        input_Q: [batch_size, len_q, d_model]\n",
        "        input_K: [batch_size, len_k, d_model]\n",
        "        input_V: [batch_size, len_v(=len_k), d_model]\n",
        "        attn_mask: [batch_size, seq_len, seq_len]\n",
        "        '''\n",
        "        residual, batch_size = input_Q, input_Q.size(0)\n",
        "        # (B, S, D) -proj-> (B, S, D_new) -split-> (B, S, H, W) -trans-> (B, H, S, W)\n",
        "        Q = self.W_Q(input_Q).view(batch_size, -1, n_heads, d_k).transpose(1,2)  # Q: [batch_size, n_heads, len_q, d_k]\n",
        "        K = self.W_K(input_K).view(batch_size, -1, n_heads, d_k).transpose(1,2)  # K: [batch_size, n_heads, len_k, d_k]\n",
        "        V = self.W_V(input_V).view(batch_size, -1, n_heads, d_v).transpose(1,2)  # V: [batch_size, n_heads, len_v(=len_k), d_v]\n",
        "\n",
        "        attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size, n_heads, seq_len, seq_len]\n",
        "\n",
        "        # context: [batch_size, n_heads, len_q, d_v], attn: [batch_size, n_heads, len_q, len_k]\n",
        "        context, attn = ScaledDotProductAttention()(Q, K, V, attn_mask)\n",
        "        context = context.transpose(1, 2).reshape(batch_size, -1, n_heads * d_v) # context: [batch_size, len_q, n_heads * d_v]\n",
        "        output = self.fc(context) # [batch_size, len_q, d_model]\n",
        "        return nn.LayerNorm(d_model).cuda()(output + residual), attn\n",
        "\n",
        "class PoswiseFeedForwardNet(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(PoswiseFeedForwardNet, self).__init__()\n",
        "        self.fc = nn.Sequential(\n",
        "            nn.Linear(d_model, d_ff, bias=False),\n",
        "            nn.ReLU(),\n",
        "            nn.Linear(d_ff, d_model, bias=False)\n",
        "        )\n",
        "    def forward(self, inputs):\n",
        "        '''\n",
        "        inputs: [batch_size, seq_len, d_model]\n",
        "        '''\n",
        "        residual = inputs\n",
        "        output = self.fc(inputs)\n",
        "        return nn.LayerNorm(d_model).cuda()(output + residual) # [batch_size, seq_len, d_model]\n",
        "\n",
        "class EncoderLayer(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(EncoderLayer, self).__init__()\n",
        "        self.enc_self_attn = MultiHeadAttention()\n",
        "        self.pos_ffn = PoswiseFeedForwardNet()\n",
        "\n",
        "    def forward(self, enc_inputs, enc_self_attn_mask):\n",
        "        '''\n",
        "        enc_inputs: [batch_size, src_len, d_model]\n",
        "        enc_self_attn_mask: [batch_size, src_len, src_len]\n",
        "        '''\n",
        "        # enc_outputs: [batch_size, src_len, d_model], attn: [batch_size, n_heads, src_len, src_len]\n",
        "        enc_outputs, attn = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs, enc_self_attn_mask) # enc_inputs to same Q,K,V\n",
        "        enc_outputs = self.pos_ffn(enc_outputs) # enc_outputs: [batch_size, src_len, d_model]\n",
        "        return enc_outputs, attn\n",
        "\n",
        "class DecoderLayer(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(DecoderLayer, self).__init__()\n",
        "        self.dec_self_attn = MultiHeadAttention()\n",
        "        self.dec_enc_attn = MultiHeadAttention()\n",
        "        self.pos_ffn = PoswiseFeedForwardNet()\n",
        "\n",
        "    def forward(self, dec_inputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask):\n",
        "        '''\n",
        "        dec_inputs: [batch_size, tgt_len, d_model]\n",
        "        enc_outputs: [batch_size, src_len, d_model]\n",
        "        dec_self_attn_mask: [batch_size, tgt_len, tgt_len]\n",
        "        dec_enc_attn_mask: [batch_size, tgt_len, src_len]\n",
        "        '''\n",
        "        # dec_outputs: [batch_size, tgt_len, d_model], dec_self_attn: [batch_size, n_heads, tgt_len, tgt_len]\n",
        "        dec_outputs, dec_self_attn = self.dec_self_attn(dec_inputs, dec_inputs, dec_inputs, dec_self_attn_mask)\n",
        "        # dec_outputs: [batch_size, tgt_len, d_model], dec_enc_attn: [batch_size, h_heads, tgt_len, src_len]\n",
        "        dec_outputs, dec_enc_attn = self.dec_enc_attn(dec_outputs, enc_outputs, enc_outputs, dec_enc_attn_mask)\n",
        "        dec_outputs = self.pos_ffn(dec_outputs) # [batch_size, tgt_len, d_model]\n",
        "        return dec_outputs, dec_self_attn, dec_enc_attn\n",
        "\n",
        "class Encoder(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Encoder, self).__init__()\n",
        "        self.src_emb = nn.Embedding(src_vocab_size, d_model)\n",
        "        self.pos_emb = PositionalEncoding(d_model)\n",
        "        self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])\n",
        "\n",
        "    def forward(self, enc_inputs):\n",
        "        '''\n",
        "        enc_inputs: [batch_size, src_len]\n",
        "        '''\n",
        "        enc_outputs = self.src_emb(enc_inputs) # [batch_size, src_len, d_model]\n",
        "        enc_outputs = self.pos_emb(enc_outputs.transpose(0, 1)).transpose(0, 1) # [batch_size, src_len, d_model]\n",
        "        enc_self_attn_mask = get_attn_pad_mask(enc_inputs, enc_inputs) # [batch_size, src_len, src_len]\n",
        "        enc_self_attns = []\n",
        "        for layer in self.layers:\n",
        "            # enc_outputs: [batch_size, src_len, d_model], enc_self_attn: [batch_size, n_heads, src_len, src_len]\n",
        "            enc_outputs, enc_self_attn = layer(enc_outputs, enc_self_attn_mask)\n",
        "            enc_self_attns.append(enc_self_attn)\n",
        "        return enc_outputs, enc_self_attns\n",
        "\n",
        "class Decoder(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Decoder, self).__init__()\n",
        "        self.tgt_emb = nn.Embedding(tgt_vocab_size, d_model)\n",
        "        self.pos_emb = PositionalEncoding(d_model)\n",
        "        self.layers = nn.ModuleList([DecoderLayer() for _ in range(n_layers)])\n",
        "\n",
        "    def forward(self, dec_inputs, enc_inputs, enc_outputs):\n",
        "        '''\n",
        "        dec_inputs: [batch_size, tgt_len]\n",
        "        enc_intpus: [batch_size, src_len]\n",
        "        enc_outputs: [batsh_size, src_len, d_model]\n",
        "        '''\n",
        "        dec_outputs = self.tgt_emb(dec_inputs) # [batch_size, tgt_len, d_model]\n",
        "        dec_outputs = self.pos_emb(dec_outputs.transpose(0, 1)).transpose(0, 1).cuda() # [batch_size, tgt_len, d_model]\n",
        "        dec_self_attn_pad_mask = get_attn_pad_mask(dec_inputs, dec_inputs).cuda() # [batch_size, tgt_len, tgt_len]\n",
        "        dec_self_attn_subsequence_mask = get_attn_subsequence_mask(dec_inputs).cuda() # [batch_size, tgt_len, tgt_len]\n",
        "        dec_self_attn_mask = torch.gt((dec_self_attn_pad_mask + dec_self_attn_subsequence_mask), 0).cuda() # [batch_size, tgt_len, tgt_len]\n",
        "\n",
        "        dec_enc_attn_mask = get_attn_pad_mask(dec_inputs, enc_inputs) # [batc_size, tgt_len, src_len]\n",
        "\n",
        "        dec_self_attns, dec_enc_attns = [], []\n",
        "        for layer in self.layers:\n",
        "            # dec_outputs: [batch_size, tgt_len, d_model], dec_self_attn: [batch_size, n_heads, tgt_len, tgt_len], dec_enc_attn: [batch_size, h_heads, tgt_len, src_len]\n",
        "            dec_outputs, dec_self_attn, dec_enc_attn = layer(dec_outputs, enc_outputs, dec_self_attn_mask, dec_enc_attn_mask)\n",
        "            dec_self_attns.append(dec_self_attn)\n",
        "            dec_enc_attns.append(dec_enc_attn)\n",
        "        return dec_outputs, dec_self_attns, dec_enc_attns\n",
        "\n",
        "class Transformer(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Transformer, self).__init__()\n",
        "        self.encoder = Encoder().cuda()\n",
        "        self.decoder = Decoder().cuda()\n",
        "        self.projection = nn.Linear(d_model, tgt_vocab_size, bias=False).cuda()\n",
        "    def forward(self, enc_inputs, dec_inputs):\n",
        "        '''\n",
        "        enc_inputs: [batch_size, src_len]\n",
        "        dec_inputs: [batch_size, tgt_len]\n",
        "        '''\n",
        "        # tensor to store decoder outputs\n",
        "        # outputs = torch.zeros(batch_size, tgt_len, tgt_vocab_size).to(self.device)\n",
        "        \n",
        "        # enc_outputs: [batch_size, src_len, d_model], enc_self_attns: [n_layers, batch_size, n_heads, src_len, src_len]\n",
        "        enc_outputs, enc_self_attns = self.encoder(enc_inputs)\n",
        "        # dec_outpus: [batch_size, tgt_len, d_model], dec_self_attns: [n_layers, batch_size, n_heads, tgt_len, tgt_len], dec_enc_attn: [n_layers, batch_size, tgt_len, src_len]\n",
        "        dec_outputs, dec_self_attns, dec_enc_attns = self.decoder(dec_inputs, enc_inputs, enc_outputs)\n",
        "        dec_logits = self.projection(dec_outputs) # dec_logits: [batch_size, tgt_len, tgt_vocab_size]\n",
        "        return dec_logits.view(-1, dec_logits.size(-1)), enc_self_attns, dec_self_attns, dec_enc_attns\n",
        "\n",
        "model = Transformer().cuda()\n",
        "criterion = nn.CrossEntropyLoss(ignore_index=0)\n",
        "optimizer = optim.SGD(model.parameters(), lr=1e-3, momentum=0.99)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nNemnO18h6PV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b305f52e-afb2-45c8-c862-54734a53533e"
      },
      "source": [
        "for epoch in range(1000):\n",
        "    for enc_inputs, dec_inputs, dec_outputs in loader:\n",
        "      '''\n",
        "      enc_inputs: [batch_size, src_len]\n",
        "      dec_inputs: [batch_size, tgt_len]\n",
        "      dec_outputs: [batch_size, tgt_len]\n",
        "      '''\n",
        "      enc_inputs, dec_inputs, dec_outputs = enc_inputs.cuda(), dec_inputs.cuda(), dec_outputs.cuda()\n",
        "      # outputs: [batch_size * tgt_len, tgt_vocab_size]\n",
        "      outputs, enc_self_attns, dec_self_attns, dec_enc_attns = model(enc_inputs, dec_inputs)\n",
        "      loss = criterion(outputs, dec_outputs.view(-1))\n",
        "      print('Epoch:', '%04d' % (epoch + 1), 'loss =', '{:.6f}'.format(loss))\n",
        "\n",
        "      optimizer.zero_grad()\n",
        "      loss.backward()\n",
        "      optimizer.step()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0001 loss = 2.155089\n",
            "Epoch: 0002 loss = 2.143922\n",
            "Epoch: 0003 loss = 1.839013\n",
            "Epoch: 0004 loss = 1.645012\n",
            "Epoch: 0005 loss = 1.427644\n",
            "Epoch: 0006 loss = 1.162449\n",
            "Epoch: 0007 loss = 1.002233\n",
            "Epoch: 0008 loss = 0.827890\n",
            "Epoch: 0009 loss = 0.614706\n",
            "Epoch: 0010 loss = 0.495598\n",
            "Epoch: 0011 loss = 0.354228\n",
            "Epoch: 0012 loss = 0.288307\n",
            "Epoch: 0013 loss = 0.208114\n",
            "Epoch: 0014 loss = 0.176501\n",
            "Epoch: 0015 loss = 0.139475\n",
            "Epoch: 0016 loss = 0.132278\n",
            "Epoch: 0017 loss = 0.105505\n",
            "Epoch: 0018 loss = 0.078170\n",
            "Epoch: 0019 loss = 0.083180\n",
            "Epoch: 0020 loss = 0.057462\n",
            "Epoch: 0021 loss = 0.062483\n",
            "Epoch: 0022 loss = 0.051939\n",
            "Epoch: 0023 loss = 0.036654\n",
            "Epoch: 0024 loss = 0.034287\n",
            "Epoch: 0025 loss = 0.034821\n",
            "Epoch: 0026 loss = 0.030769\n",
            "Epoch: 0027 loss = 0.020328\n",
            "Epoch: 0028 loss = 0.020767\n",
            "Epoch: 0029 loss = 0.018654\n",
            "Epoch: 0030 loss = 0.013562\n",
            "Epoch: 0031 loss = 0.011509\n",
            "Epoch: 0032 loss = 0.017865\n",
            "Epoch: 0033 loss = 0.014290\n",
            "Epoch: 0034 loss = 0.009512\n",
            "Epoch: 0035 loss = 0.010811\n",
            "Epoch: 0036 loss = 0.008564\n",
            "Epoch: 0037 loss = 0.004911\n",
            "Epoch: 0038 loss = 0.007077\n",
            "Epoch: 0039 loss = 0.006047\n",
            "Epoch: 0040 loss = 0.005938\n",
            "Epoch: 0041 loss = 0.003831\n",
            "Epoch: 0042 loss = 0.006380\n",
            "Epoch: 0043 loss = 0.002963\n",
            "Epoch: 0044 loss = 0.002480\n",
            "Epoch: 0045 loss = 0.002371\n",
            "Epoch: 0046 loss = 0.002465\n",
            "Epoch: 0047 loss = 0.003212\n",
            "Epoch: 0048 loss = 0.002674\n",
            "Epoch: 0049 loss = 0.002124\n",
            "Epoch: 0050 loss = 0.001626\n",
            "Epoch: 0051 loss = 0.001504\n",
            "Epoch: 0052 loss = 0.001762\n",
            "Epoch: 0053 loss = 0.001107\n",
            "Epoch: 0054 loss = 0.001317\n",
            "Epoch: 0055 loss = 0.000912\n",
            "Epoch: 0056 loss = 0.000989\n",
            "Epoch: 0057 loss = 0.000732\n",
            "Epoch: 0058 loss = 0.001100\n",
            "Epoch: 0059 loss = 0.001123\n",
            "Epoch: 0060 loss = 0.000934\n",
            "Epoch: 0061 loss = 0.001175\n",
            "Epoch: 0062 loss = 0.001276\n",
            "Epoch: 0063 loss = 0.001021\n",
            "Epoch: 0064 loss = 0.001153\n",
            "Epoch: 0065 loss = 0.000615\n",
            "Epoch: 0066 loss = 0.000729\n",
            "Epoch: 0067 loss = 0.000826\n",
            "Epoch: 0068 loss = 0.000954\n",
            "Epoch: 0069 loss = 0.000836\n",
            "Epoch: 0070 loss = 0.001357\n",
            "Epoch: 0071 loss = 0.001331\n",
            "Epoch: 0072 loss = 0.000815\n",
            "Epoch: 0073 loss = 0.001017\n",
            "Epoch: 0074 loss = 0.001259\n",
            "Epoch: 0075 loss = 0.001370\n",
            "Epoch: 0076 loss = 0.001418\n",
            "Epoch: 0077 loss = 0.002051\n",
            "Epoch: 0078 loss = 0.001119\n",
            "Epoch: 0079 loss = 0.001790\n",
            "Epoch: 0080 loss = 0.001487\n",
            "Epoch: 0081 loss = 0.001337\n",
            "Epoch: 0082 loss = 0.001047\n",
            "Epoch: 0083 loss = 0.001386\n",
            "Epoch: 0084 loss = 0.001014\n",
            "Epoch: 0085 loss = 0.001326\n",
            "Epoch: 0086 loss = 0.001097\n",
            "Epoch: 0087 loss = 0.001219\n",
            "Epoch: 0088 loss = 0.000795\n",
            "Epoch: 0089 loss = 0.001643\n",
            "Epoch: 0090 loss = 0.001264\n",
            "Epoch: 0091 loss = 0.001353\n",
            "Epoch: 0092 loss = 0.000615\n",
            "Epoch: 0093 loss = 0.000584\n",
            "Epoch: 0094 loss = 0.000612\n",
            "Epoch: 0095 loss = 0.000372\n",
            "Epoch: 0096 loss = 0.000501\n",
            "Epoch: 0097 loss = 0.000614\n",
            "Epoch: 0098 loss = 0.000446\n",
            "Epoch: 0099 loss = 0.000489\n",
            "Epoch: 0100 loss = 0.000572\n",
            "Epoch: 0101 loss = 0.000334\n",
            "Epoch: 0102 loss = 0.000311\n",
            "Epoch: 0103 loss = 0.000455\n",
            "Epoch: 0104 loss = 0.000243\n",
            "Epoch: 0105 loss = 0.000271\n",
            "Epoch: 0106 loss = 0.000252\n",
            "Epoch: 0107 loss = 0.000273\n",
            "Epoch: 0108 loss = 0.000235\n",
            "Epoch: 0109 loss = 0.000159\n",
            "Epoch: 0110 loss = 0.000197\n",
            "Epoch: 0111 loss = 0.000150\n",
            "Epoch: 0112 loss = 0.000128\n",
            "Epoch: 0113 loss = 0.000115\n",
            "Epoch: 0114 loss = 0.000132\n",
            "Epoch: 0115 loss = 0.000113\n",
            "Epoch: 0116 loss = 0.000134\n",
            "Epoch: 0117 loss = 0.000098\n",
            "Epoch: 0118 loss = 0.000088\n",
            "Epoch: 0119 loss = 0.000096\n",
            "Epoch: 0120 loss = 0.000079\n",
            "Epoch: 0121 loss = 0.000076\n",
            "Epoch: 0122 loss = 0.000076\n",
            "Epoch: 0123 loss = 0.000089\n",
            "Epoch: 0124 loss = 0.000059\n",
            "Epoch: 0125 loss = 0.000091\n",
            "Epoch: 0126 loss = 0.000066\n",
            "Epoch: 0127 loss = 0.000070\n",
            "Epoch: 0128 loss = 0.000060\n",
            "Epoch: 0129 loss = 0.000056\n",
            "Epoch: 0130 loss = 0.000058\n",
            "Epoch: 0131 loss = 0.000069\n",
            "Epoch: 0132 loss = 0.000045\n",
            "Epoch: 0133 loss = 0.000074\n",
            "Epoch: 0134 loss = 0.000046\n",
            "Epoch: 0135 loss = 0.000046\n",
            "Epoch: 0136 loss = 0.000040\n",
            "Epoch: 0137 loss = 0.000043\n",
            "Epoch: 0138 loss = 0.000048\n",
            "Epoch: 0139 loss = 0.000034\n",
            "Epoch: 0140 loss = 0.000031\n",
            "Epoch: 0141 loss = 0.000032\n",
            "Epoch: 0142 loss = 0.000027\n",
            "Epoch: 0143 loss = 0.000027\n",
            "Epoch: 0144 loss = 0.000030\n",
            "Epoch: 0145 loss = 0.000030\n",
            "Epoch: 0146 loss = 0.000036\n",
            "Epoch: 0147 loss = 0.000030\n",
            "Epoch: 0148 loss = 0.000040\n",
            "Epoch: 0149 loss = 0.000030\n",
            "Epoch: 0150 loss = 0.000035\n",
            "Epoch: 0151 loss = 0.000029\n",
            "Epoch: 0152 loss = 0.000027\n",
            "Epoch: 0153 loss = 0.000029\n",
            "Epoch: 0154 loss = 0.000023\n",
            "Epoch: 0155 loss = 0.000019\n",
            "Epoch: 0156 loss = 0.000026\n",
            "Epoch: 0157 loss = 0.000031\n",
            "Epoch: 0158 loss = 0.000033\n",
            "Epoch: 0159 loss = 0.000020\n",
            "Epoch: 0160 loss = 0.000020\n",
            "Epoch: 0161 loss = 0.000024\n",
            "Epoch: 0162 loss = 0.000025\n",
            "Epoch: 0163 loss = 0.000036\n",
            "Epoch: 0164 loss = 0.000020\n",
            "Epoch: 0165 loss = 0.000028\n",
            "Epoch: 0166 loss = 0.000024\n",
            "Epoch: 0167 loss = 0.000021\n",
            "Epoch: 0168 loss = 0.000027\n",
            "Epoch: 0169 loss = 0.000023\n",
            "Epoch: 0170 loss = 0.000023\n",
            "Epoch: 0171 loss = 0.000031\n",
            "Epoch: 0172 loss = 0.000021\n",
            "Epoch: 0173 loss = 0.000030\n",
            "Epoch: 0174 loss = 0.000022\n",
            "Epoch: 0175 loss = 0.000030\n",
            "Epoch: 0176 loss = 0.000018\n",
            "Epoch: 0177 loss = 0.000028\n",
            "Epoch: 0178 loss = 0.000024\n",
            "Epoch: 0179 loss = 0.000032\n",
            "Epoch: 0180 loss = 0.000016\n",
            "Epoch: 0181 loss = 0.000022\n",
            "Epoch: 0182 loss = 0.000034\n",
            "Epoch: 0183 loss = 0.000023\n",
            "Epoch: 0184 loss = 0.000024\n",
            "Epoch: 0185 loss = 0.000024\n",
            "Epoch: 0186 loss = 0.000018\n",
            "Epoch: 0187 loss = 0.000020\n",
            "Epoch: 0188 loss = 0.000025\n",
            "Epoch: 0189 loss = 0.000020\n",
            "Epoch: 0190 loss = 0.000025\n",
            "Epoch: 0191 loss = 0.000038\n",
            "Epoch: 0192 loss = 0.000016\n",
            "Epoch: 0193 loss = 0.000023\n",
            "Epoch: 0194 loss = 0.000017\n",
            "Epoch: 0195 loss = 0.000020\n",
            "Epoch: 0196 loss = 0.000018\n",
            "Epoch: 0197 loss = 0.000015\n",
            "Epoch: 0198 loss = 0.000024\n",
            "Epoch: 0199 loss = 0.000024\n",
            "Epoch: 0200 loss = 0.000026\n",
            "Epoch: 0201 loss = 0.000018\n",
            "Epoch: 0202 loss = 0.000026\n",
            "Epoch: 0203 loss = 0.000017\n",
            "Epoch: 0204 loss = 0.000022\n",
            "Epoch: 0205 loss = 0.000017\n",
            "Epoch: 0206 loss = 0.000022\n",
            "Epoch: 0207 loss = 0.000029\n",
            "Epoch: 0208 loss = 0.000020\n",
            "Epoch: 0209 loss = 0.000015\n",
            "Epoch: 0210 loss = 0.000019\n",
            "Epoch: 0211 loss = 0.000031\n",
            "Epoch: 0212 loss = 0.000027\n",
            "Epoch: 0213 loss = 0.000018\n",
            "Epoch: 0214 loss = 0.000030\n",
            "Epoch: 0215 loss = 0.000021\n",
            "Epoch: 0216 loss = 0.000024\n",
            "Epoch: 0217 loss = 0.000019\n",
            "Epoch: 0218 loss = 0.000037\n",
            "Epoch: 0219 loss = 0.000029\n",
            "Epoch: 0220 loss = 0.000019\n",
            "Epoch: 0221 loss = 0.000017\n",
            "Epoch: 0222 loss = 0.000026\n",
            "Epoch: 0223 loss = 0.000019\n",
            "Epoch: 0224 loss = 0.000018\n",
            "Epoch: 0225 loss = 0.000012\n",
            "Epoch: 0226 loss = 0.000023\n",
            "Epoch: 0227 loss = 0.000029\n",
            "Epoch: 0228 loss = 0.000020\n",
            "Epoch: 0229 loss = 0.000027\n",
            "Epoch: 0230 loss = 0.000018\n",
            "Epoch: 0231 loss = 0.000015\n",
            "Epoch: 0232 loss = 0.000024\n",
            "Epoch: 0233 loss = 0.000024\n",
            "Epoch: 0234 loss = 0.000024\n",
            "Epoch: 0235 loss = 0.000024\n",
            "Epoch: 0236 loss = 0.000027\n",
            "Epoch: 0237 loss = 0.000036\n",
            "Epoch: 0238 loss = 0.000011\n",
            "Epoch: 0239 loss = 0.000024\n",
            "Epoch: 0240 loss = 0.000024\n",
            "Epoch: 0241 loss = 0.000019\n",
            "Epoch: 0242 loss = 0.000032\n",
            "Epoch: 0243 loss = 0.000025\n",
            "Epoch: 0244 loss = 0.000045\n",
            "Epoch: 0245 loss = 0.000017\n",
            "Epoch: 0246 loss = 0.000022\n",
            "Epoch: 0247 loss = 0.000030\n",
            "Epoch: 0248 loss = 0.000040\n",
            "Epoch: 0249 loss = 0.000017\n",
            "Epoch: 0250 loss = 0.000027\n",
            "Epoch: 0251 loss = 0.000028\n",
            "Epoch: 0252 loss = 0.000029\n",
            "Epoch: 0253 loss = 0.000034\n",
            "Epoch: 0254 loss = 0.000029\n",
            "Epoch: 0255 loss = 0.000019\n",
            "Epoch: 0256 loss = 0.000021\n",
            "Epoch: 0257 loss = 0.000026\n",
            "Epoch: 0258 loss = 0.000039\n",
            "Epoch: 0259 loss = 0.000034\n",
            "Epoch: 0260 loss = 0.000022\n",
            "Epoch: 0261 loss = 0.000029\n",
            "Epoch: 0262 loss = 0.000033\n",
            "Epoch: 0263 loss = 0.000031\n",
            "Epoch: 0264 loss = 0.000024\n",
            "Epoch: 0265 loss = 0.000024\n",
            "Epoch: 0266 loss = 0.000030\n",
            "Epoch: 0267 loss = 0.000020\n",
            "Epoch: 0268 loss = 0.000023\n",
            "Epoch: 0269 loss = 0.000035\n",
            "Epoch: 0270 loss = 0.000044\n",
            "Epoch: 0271 loss = 0.000032\n",
            "Epoch: 0272 loss = 0.000025\n",
            "Epoch: 0273 loss = 0.000043\n",
            "Epoch: 0274 loss = 0.000022\n",
            "Epoch: 0275 loss = 0.000064\n",
            "Epoch: 0276 loss = 0.000013\n",
            "Epoch: 0277 loss = 0.000031\n",
            "Epoch: 0278 loss = 0.000017\n",
            "Epoch: 0279 loss = 0.000035\n",
            "Epoch: 0280 loss = 0.000023\n",
            "Epoch: 0281 loss = 0.000035\n",
            "Epoch: 0282 loss = 0.000050\n",
            "Epoch: 0283 loss = 0.000038\n",
            "Epoch: 0284 loss = 0.000027\n",
            "Epoch: 0285 loss = 0.000020\n",
            "Epoch: 0286 loss = 0.000022\n",
            "Epoch: 0287 loss = 0.000026\n",
            "Epoch: 0288 loss = 0.000022\n",
            "Epoch: 0289 loss = 0.000050\n",
            "Epoch: 0290 loss = 0.000024\n",
            "Epoch: 0291 loss = 0.000030\n",
            "Epoch: 0292 loss = 0.000056\n",
            "Epoch: 0293 loss = 0.000021\n",
            "Epoch: 0294 loss = 0.000073\n",
            "Epoch: 0295 loss = 0.000019\n",
            "Epoch: 0296 loss = 0.000024\n",
            "Epoch: 0297 loss = 0.000035\n",
            "Epoch: 0298 loss = 0.000036\n",
            "Epoch: 0299 loss = 0.000052\n",
            "Epoch: 0300 loss = 0.000038\n",
            "Epoch: 0301 loss = 0.000052\n",
            "Epoch: 0302 loss = 0.000013\n",
            "Epoch: 0303 loss = 0.000044\n",
            "Epoch: 0304 loss = 0.000026\n",
            "Epoch: 0305 loss = 0.000052\n",
            "Epoch: 0306 loss = 0.000045\n",
            "Epoch: 0307 loss = 0.000019\n",
            "Epoch: 0308 loss = 0.000038\n",
            "Epoch: 0309 loss = 0.000046\n",
            "Epoch: 0310 loss = 0.000016\n",
            "Epoch: 0311 loss = 0.000025\n",
            "Epoch: 0312 loss = 0.000033\n",
            "Epoch: 0313 loss = 0.000047\n",
            "Epoch: 0314 loss = 0.000028\n",
            "Epoch: 0315 loss = 0.000052\n",
            "Epoch: 0316 loss = 0.000027\n",
            "Epoch: 0317 loss = 0.000049\n",
            "Epoch: 0318 loss = 0.000025\n",
            "Epoch: 0319 loss = 0.000043\n",
            "Epoch: 0320 loss = 0.000104\n",
            "Epoch: 0321 loss = 0.000025\n",
            "Epoch: 0322 loss = 0.000060\n",
            "Epoch: 0323 loss = 0.000029\n",
            "Epoch: 0324 loss = 0.000041\n",
            "Epoch: 0325 loss = 0.000017\n",
            "Epoch: 0326 loss = 0.000021\n",
            "Epoch: 0327 loss = 0.000040\n",
            "Epoch: 0328 loss = 0.000041\n",
            "Epoch: 0329 loss = 0.000031\n",
            "Epoch: 0330 loss = 0.000035\n",
            "Epoch: 0331 loss = 0.000035\n",
            "Epoch: 0332 loss = 0.000039\n",
            "Epoch: 0333 loss = 0.000040\n",
            "Epoch: 0334 loss = 0.000046\n",
            "Epoch: 0335 loss = 0.000023\n",
            "Epoch: 0336 loss = 0.000061\n",
            "Epoch: 0337 loss = 0.000024\n",
            "Epoch: 0338 loss = 0.000038\n",
            "Epoch: 0339 loss = 0.000073\n",
            "Epoch: 0340 loss = 0.000046\n",
            "Epoch: 0341 loss = 0.000037\n",
            "Epoch: 0342 loss = 0.000023\n",
            "Epoch: 0343 loss = 0.000032\n",
            "Epoch: 0344 loss = 0.000051\n",
            "Epoch: 0345 loss = 0.000017\n",
            "Epoch: 0346 loss = 0.000028\n",
            "Epoch: 0347 loss = 0.000049\n",
            "Epoch: 0348 loss = 0.000029\n",
            "Epoch: 0349 loss = 0.000034\n",
            "Epoch: 0350 loss = 0.000026\n",
            "Epoch: 0351 loss = 0.000033\n",
            "Epoch: 0352 loss = 0.000040\n",
            "Epoch: 0353 loss = 0.000050\n",
            "Epoch: 0354 loss = 0.000023\n",
            "Epoch: 0355 loss = 0.000048\n",
            "Epoch: 0356 loss = 0.000025\n",
            "Epoch: 0357 loss = 0.000019\n",
            "Epoch: 0358 loss = 0.000024\n",
            "Epoch: 0359 loss = 0.000044\n",
            "Epoch: 0360 loss = 0.000049\n",
            "Epoch: 0361 loss = 0.000053\n",
            "Epoch: 0362 loss = 0.000071\n",
            "Epoch: 0363 loss = 0.000054\n",
            "Epoch: 0364 loss = 0.000016\n",
            "Epoch: 0365 loss = 0.000020\n",
            "Epoch: 0366 loss = 0.000025\n",
            "Epoch: 0367 loss = 0.000048\n",
            "Epoch: 0368 loss = 0.000025\n",
            "Epoch: 0369 loss = 0.000039\n",
            "Epoch: 0370 loss = 0.000027\n",
            "Epoch: 0371 loss = 0.000015\n",
            "Epoch: 0372 loss = 0.000023\n",
            "Epoch: 0373 loss = 0.000034\n",
            "Epoch: 0374 loss = 0.000030\n",
            "Epoch: 0375 loss = 0.000089\n",
            "Epoch: 0376 loss = 0.000022\n",
            "Epoch: 0377 loss = 0.000051\n",
            "Epoch: 0378 loss = 0.000024\n",
            "Epoch: 0379 loss = 0.000013\n",
            "Epoch: 0380 loss = 0.000049\n",
            "Epoch: 0381 loss = 0.000047\n",
            "Epoch: 0382 loss = 0.000029\n",
            "Epoch: 0383 loss = 0.000017\n",
            "Epoch: 0384 loss = 0.000038\n",
            "Epoch: 0385 loss = 0.000014\n",
            "Epoch: 0386 loss = 0.000015\n",
            "Epoch: 0387 loss = 0.000028\n",
            "Epoch: 0388 loss = 0.000022\n",
            "Epoch: 0389 loss = 0.000030\n",
            "Epoch: 0390 loss = 0.000022\n",
            "Epoch: 0391 loss = 0.000021\n",
            "Epoch: 0392 loss = 0.000018\n",
            "Epoch: 0393 loss = 0.000072\n",
            "Epoch: 0394 loss = 0.000023\n",
            "Epoch: 0395 loss = 0.000039\n",
            "Epoch: 0396 loss = 0.000010\n",
            "Epoch: 0397 loss = 0.000021\n",
            "Epoch: 0398 loss = 0.000021\n",
            "Epoch: 0399 loss = 0.000022\n",
            "Epoch: 0400 loss = 0.000025\n",
            "Epoch: 0401 loss = 0.000012\n",
            "Epoch: 0402 loss = 0.000027\n",
            "Epoch: 0403 loss = 0.000068\n",
            "Epoch: 0404 loss = 0.000019\n",
            "Epoch: 0405 loss = 0.000020\n",
            "Epoch: 0406 loss = 0.000016\n",
            "Epoch: 0407 loss = 0.000024\n",
            "Epoch: 0408 loss = 0.000030\n",
            "Epoch: 0409 loss = 0.000065\n",
            "Epoch: 0410 loss = 0.000038\n",
            "Epoch: 0411 loss = 0.000018\n",
            "Epoch: 0412 loss = 0.000017\n",
            "Epoch: 0413 loss = 0.000020\n",
            "Epoch: 0414 loss = 0.000019\n",
            "Epoch: 0415 loss = 0.000028\n",
            "Epoch: 0416 loss = 0.000018\n",
            "Epoch: 0417 loss = 0.000010\n",
            "Epoch: 0418 loss = 0.000008\n",
            "Epoch: 0419 loss = 0.000016\n",
            "Epoch: 0420 loss = 0.000013\n",
            "Epoch: 0421 loss = 0.000015\n",
            "Epoch: 0422 loss = 0.000052\n",
            "Epoch: 0423 loss = 0.000028\n",
            "Epoch: 0424 loss = 0.000014\n",
            "Epoch: 0425 loss = 0.000019\n",
            "Epoch: 0426 loss = 0.000016\n",
            "Epoch: 0427 loss = 0.000023\n",
            "Epoch: 0428 loss = 0.000011\n",
            "Epoch: 0429 loss = 0.000017\n",
            "Epoch: 0430 loss = 0.000019\n",
            "Epoch: 0431 loss = 0.000021\n",
            "Epoch: 0432 loss = 0.000048\n",
            "Epoch: 0433 loss = 0.000015\n",
            "Epoch: 0434 loss = 0.000020\n",
            "Epoch: 0435 loss = 0.000018\n",
            "Epoch: 0436 loss = 0.000022\n",
            "Epoch: 0437 loss = 0.000024\n",
            "Epoch: 0438 loss = 0.000021\n",
            "Epoch: 0439 loss = 0.000021\n",
            "Epoch: 0440 loss = 0.000019\n",
            "Epoch: 0441 loss = 0.000011\n",
            "Epoch: 0442 loss = 0.000018\n",
            "Epoch: 0443 loss = 0.000025\n",
            "Epoch: 0444 loss = 0.000017\n",
            "Epoch: 0445 loss = 0.000032\n",
            "Epoch: 0446 loss = 0.000017\n",
            "Epoch: 0447 loss = 0.000011\n",
            "Epoch: 0448 loss = 0.000011\n",
            "Epoch: 0449 loss = 0.000033\n",
            "Epoch: 0450 loss = 0.000017\n",
            "Epoch: 0451 loss = 0.000023\n",
            "Epoch: 0452 loss = 0.000019\n",
            "Epoch: 0453 loss = 0.000013\n",
            "Epoch: 0454 loss = 0.000016\n",
            "Epoch: 0455 loss = 0.000019\n",
            "Epoch: 0456 loss = 0.000011\n",
            "Epoch: 0457 loss = 0.000021\n",
            "Epoch: 0458 loss = 0.000020\n",
            "Epoch: 0459 loss = 0.000016\n",
            "Epoch: 0460 loss = 0.000035\n",
            "Epoch: 0461 loss = 0.000016\n",
            "Epoch: 0462 loss = 0.000010\n",
            "Epoch: 0463 loss = 0.000014\n",
            "Epoch: 0464 loss = 0.000014\n",
            "Epoch: 0465 loss = 0.000019\n",
            "Epoch: 0466 loss = 0.000013\n",
            "Epoch: 0467 loss = 0.000013\n",
            "Epoch: 0468 loss = 0.000013\n",
            "Epoch: 0469 loss = 0.000028\n",
            "Epoch: 0470 loss = 0.000011\n",
            "Epoch: 0471 loss = 0.000017\n",
            "Epoch: 0472 loss = 0.000013\n",
            "Epoch: 0473 loss = 0.000019\n",
            "Epoch: 0474 loss = 0.000022\n",
            "Epoch: 0475 loss = 0.000012\n",
            "Epoch: 0476 loss = 0.000015\n",
            "Epoch: 0477 loss = 0.000014\n",
            "Epoch: 0478 loss = 0.000015\n",
            "Epoch: 0479 loss = 0.000013\n",
            "Epoch: 0480 loss = 0.000010\n",
            "Epoch: 0481 loss = 0.000014\n",
            "Epoch: 0482 loss = 0.000010\n",
            "Epoch: 0483 loss = 0.000013\n",
            "Epoch: 0484 loss = 0.000015\n",
            "Epoch: 0485 loss = 0.000014\n",
            "Epoch: 0486 loss = 0.000024\n",
            "Epoch: 0487 loss = 0.000016\n",
            "Epoch: 0488 loss = 0.000009\n",
            "Epoch: 0489 loss = 0.000020\n",
            "Epoch: 0490 loss = 0.000019\n",
            "Epoch: 0491 loss = 0.000011\n",
            "Epoch: 0492 loss = 0.000009\n",
            "Epoch: 0493 loss = 0.000013\n",
            "Epoch: 0494 loss = 0.000014\n",
            "Epoch: 0495 loss = 0.000009\n",
            "Epoch: 0496 loss = 0.000011\n",
            "Epoch: 0497 loss = 0.000010\n",
            "Epoch: 0498 loss = 0.000010\n",
            "Epoch: 0499 loss = 0.000011\n",
            "Epoch: 0500 loss = 0.000010\n",
            "Epoch: 0501 loss = 0.000015\n",
            "Epoch: 0502 loss = 0.000011\n",
            "Epoch: 0503 loss = 0.000012\n",
            "Epoch: 0504 loss = 0.000011\n",
            "Epoch: 0505 loss = 0.000013\n",
            "Epoch: 0506 loss = 0.000008\n",
            "Epoch: 0507 loss = 0.000011\n",
            "Epoch: 0508 loss = 0.000022\n",
            "Epoch: 0509 loss = 0.000008\n",
            "Epoch: 0510 loss = 0.000008\n",
            "Epoch: 0511 loss = 0.000008\n",
            "Epoch: 0512 loss = 0.000010\n",
            "Epoch: 0513 loss = 0.000016\n",
            "Epoch: 0514 loss = 0.000014\n",
            "Epoch: 0515 loss = 0.000006\n",
            "Epoch: 0516 loss = 0.000015\n",
            "Epoch: 0517 loss = 0.000013\n",
            "Epoch: 0518 loss = 0.000007\n",
            "Epoch: 0519 loss = 0.000005\n",
            "Epoch: 0520 loss = 0.000009\n",
            "Epoch: 0521 loss = 0.000022\n",
            "Epoch: 0522 loss = 0.000010\n",
            "Epoch: 0523 loss = 0.000009\n",
            "Epoch: 0524 loss = 0.000009\n",
            "Epoch: 0525 loss = 0.000014\n",
            "Epoch: 0526 loss = 0.000018\n",
            "Epoch: 0527 loss = 0.000007\n",
            "Epoch: 0528 loss = 0.000005\n",
            "Epoch: 0529 loss = 0.000009\n",
            "Epoch: 0530 loss = 0.000009\n",
            "Epoch: 0531 loss = 0.000005\n",
            "Epoch: 0532 loss = 0.000013\n",
            "Epoch: 0533 loss = 0.000008\n",
            "Epoch: 0534 loss = 0.000011\n",
            "Epoch: 0535 loss = 0.000010\n",
            "Epoch: 0536 loss = 0.000007\n",
            "Epoch: 0537 loss = 0.000010\n",
            "Epoch: 0538 loss = 0.000007\n",
            "Epoch: 0539 loss = 0.000009\n",
            "Epoch: 0540 loss = 0.000009\n",
            "Epoch: 0541 loss = 0.000005\n",
            "Epoch: 0542 loss = 0.000006\n",
            "Epoch: 0543 loss = 0.000011\n",
            "Epoch: 0544 loss = 0.000012\n",
            "Epoch: 0545 loss = 0.000006\n",
            "Epoch: 0546 loss = 0.000013\n",
            "Epoch: 0547 loss = 0.000008\n",
            "Epoch: 0548 loss = 0.000011\n",
            "Epoch: 0549 loss = 0.000008\n",
            "Epoch: 0550 loss = 0.000007\n",
            "Epoch: 0551 loss = 0.000013\n",
            "Epoch: 0552 loss = 0.000005\n",
            "Epoch: 0553 loss = 0.000007\n",
            "Epoch: 0554 loss = 0.000009\n",
            "Epoch: 0555 loss = 0.000011\n",
            "Epoch: 0556 loss = 0.000012\n",
            "Epoch: 0557 loss = 0.000006\n",
            "Epoch: 0558 loss = 0.000016\n",
            "Epoch: 0559 loss = 0.000008\n",
            "Epoch: 0560 loss = 0.000011\n",
            "Epoch: 0561 loss = 0.000009\n",
            "Epoch: 0562 loss = 0.000009\n",
            "Epoch: 0563 loss = 0.000006\n",
            "Epoch: 0564 loss = 0.000009\n",
            "Epoch: 0565 loss = 0.000010\n",
            "Epoch: 0566 loss = 0.000010\n",
            "Epoch: 0567 loss = 0.000011\n",
            "Epoch: 0568 loss = 0.000012\n",
            "Epoch: 0569 loss = 0.000006\n",
            "Epoch: 0570 loss = 0.000023\n",
            "Epoch: 0571 loss = 0.000006\n",
            "Epoch: 0572 loss = 0.000005\n",
            "Epoch: 0573 loss = 0.000007\n",
            "Epoch: 0574 loss = 0.000008\n",
            "Epoch: 0575 loss = 0.000014\n",
            "Epoch: 0576 loss = 0.000009\n",
            "Epoch: 0577 loss = 0.000010\n",
            "Epoch: 0578 loss = 0.000011\n",
            "Epoch: 0579 loss = 0.000005\n",
            "Epoch: 0580 loss = 0.000008\n",
            "Epoch: 0581 loss = 0.000005\n",
            "Epoch: 0582 loss = 0.000013\n",
            "Epoch: 0583 loss = 0.000006\n",
            "Epoch: 0584 loss = 0.000010\n",
            "Epoch: 0585 loss = 0.000007\n",
            "Epoch: 0586 loss = 0.000005\n",
            "Epoch: 0587 loss = 0.000008\n",
            "Epoch: 0588 loss = 0.000010\n",
            "Epoch: 0589 loss = 0.000005\n",
            "Epoch: 0590 loss = 0.000009\n",
            "Epoch: 0591 loss = 0.000007\n",
            "Epoch: 0592 loss = 0.000006\n",
            "Epoch: 0593 loss = 0.000005\n",
            "Epoch: 0594 loss = 0.000006\n",
            "Epoch: 0595 loss = 0.000012\n",
            "Epoch: 0596 loss = 0.000014\n",
            "Epoch: 0597 loss = 0.000010\n",
            "Epoch: 0598 loss = 0.000005\n",
            "Epoch: 0599 loss = 0.000006\n",
            "Epoch: 0600 loss = 0.000009\n",
            "Epoch: 0601 loss = 0.000006\n",
            "Epoch: 0602 loss = 0.000005\n",
            "Epoch: 0603 loss = 0.000010\n",
            "Epoch: 0604 loss = 0.000007\n",
            "Epoch: 0605 loss = 0.000009\n",
            "Epoch: 0606 loss = 0.000006\n",
            "Epoch: 0607 loss = 0.000012\n",
            "Epoch: 0608 loss = 0.000007\n",
            "Epoch: 0609 loss = 0.000008\n",
            "Epoch: 0610 loss = 0.000010\n",
            "Epoch: 0611 loss = 0.000008\n",
            "Epoch: 0612 loss = 0.000009\n",
            "Epoch: 0613 loss = 0.000006\n",
            "Epoch: 0614 loss = 0.000014\n",
            "Epoch: 0615 loss = 0.000006\n",
            "Epoch: 0616 loss = 0.000004\n",
            "Epoch: 0617 loss = 0.000008\n",
            "Epoch: 0618 loss = 0.000005\n",
            "Epoch: 0619 loss = 0.000011\n",
            "Epoch: 0620 loss = 0.000008\n",
            "Epoch: 0621 loss = 0.000007\n",
            "Epoch: 0622 loss = 0.000008\n",
            "Epoch: 0623 loss = 0.000009\n",
            "Epoch: 0624 loss = 0.000008\n",
            "Epoch: 0625 loss = 0.000008\n",
            "Epoch: 0626 loss = 0.000010\n",
            "Epoch: 0627 loss = 0.000004\n",
            "Epoch: 0628 loss = 0.000009\n",
            "Epoch: 0629 loss = 0.000007\n",
            "Epoch: 0630 loss = 0.000008\n",
            "Epoch: 0631 loss = 0.000006\n",
            "Epoch: 0632 loss = 0.000007\n",
            "Epoch: 0633 loss = 0.000006\n",
            "Epoch: 0634 loss = 0.000004\n",
            "Epoch: 0635 loss = 0.000005\n",
            "Epoch: 0636 loss = 0.000007\n",
            "Epoch: 0637 loss = 0.000009\n",
            "Epoch: 0638 loss = 0.000010\n",
            "Epoch: 0639 loss = 0.000005\n",
            "Epoch: 0640 loss = 0.000008\n",
            "Epoch: 0641 loss = 0.000007\n",
            "Epoch: 0642 loss = 0.000004\n",
            "Epoch: 0643 loss = 0.000003\n",
            "Epoch: 0644 loss = 0.000004\n",
            "Epoch: 0645 loss = 0.000008\n",
            "Epoch: 0646 loss = 0.000008\n",
            "Epoch: 0647 loss = 0.000011\n",
            "Epoch: 0648 loss = 0.000008\n",
            "Epoch: 0649 loss = 0.000005\n",
            "Epoch: 0650 loss = 0.000008\n",
            "Epoch: 0651 loss = 0.000009\n",
            "Epoch: 0652 loss = 0.000004\n",
            "Epoch: 0653 loss = 0.000007\n",
            "Epoch: 0654 loss = 0.000007\n",
            "Epoch: 0655 loss = 0.000004\n",
            "Epoch: 0656 loss = 0.000017\n",
            "Epoch: 0657 loss = 0.000005\n",
            "Epoch: 0658 loss = 0.000007\n",
            "Epoch: 0659 loss = 0.000009\n",
            "Epoch: 0660 loss = 0.000006\n",
            "Epoch: 0661 loss = 0.000009\n",
            "Epoch: 0662 loss = 0.000003\n",
            "Epoch: 0663 loss = 0.000005\n",
            "Epoch: 0664 loss = 0.000006\n",
            "Epoch: 0665 loss = 0.000004\n",
            "Epoch: 0666 loss = 0.000007\n",
            "Epoch: 0667 loss = 0.000009\n",
            "Epoch: 0668 loss = 0.000004\n",
            "Epoch: 0669 loss = 0.000006\n",
            "Epoch: 0670 loss = 0.000006\n",
            "Epoch: 0671 loss = 0.000002\n",
            "Epoch: 0672 loss = 0.000005\n",
            "Epoch: 0673 loss = 0.000004\n",
            "Epoch: 0674 loss = 0.000004\n",
            "Epoch: 0675 loss = 0.000008\n",
            "Epoch: 0676 loss = 0.000006\n",
            "Epoch: 0677 loss = 0.000006\n",
            "Epoch: 0678 loss = 0.000006\n",
            "Epoch: 0679 loss = 0.000011\n",
            "Epoch: 0680 loss = 0.000004\n",
            "Epoch: 0681 loss = 0.000005\n",
            "Epoch: 0682 loss = 0.000006\n",
            "Epoch: 0683 loss = 0.000004\n",
            "Epoch: 0684 loss = 0.000006\n",
            "Epoch: 0685 loss = 0.000004\n",
            "Epoch: 0686 loss = 0.000004\n",
            "Epoch: 0687 loss = 0.000006\n",
            "Epoch: 0688 loss = 0.000004\n",
            "Epoch: 0689 loss = 0.000004\n",
            "Epoch: 0690 loss = 0.000003\n",
            "Epoch: 0691 loss = 0.000003\n",
            "Epoch: 0692 loss = 0.000007\n",
            "Epoch: 0693 loss = 0.000007\n",
            "Epoch: 0694 loss = 0.000007\n",
            "Epoch: 0695 loss = 0.000004\n",
            "Epoch: 0696 loss = 0.000004\n",
            "Epoch: 0697 loss = 0.000003\n",
            "Epoch: 0698 loss = 0.000005\n",
            "Epoch: 0699 loss = 0.000003\n",
            "Epoch: 0700 loss = 0.000006\n",
            "Epoch: 0701 loss = 0.000006\n",
            "Epoch: 0702 loss = 0.000003\n",
            "Epoch: 0703 loss = 0.000003\n",
            "Epoch: 0704 loss = 0.000007\n",
            "Epoch: 0705 loss = 0.000006\n",
            "Epoch: 0706 loss = 0.000005\n",
            "Epoch: 0707 loss = 0.000004\n",
            "Epoch: 0708 loss = 0.000006\n",
            "Epoch: 0709 loss = 0.000006\n",
            "Epoch: 0710 loss = 0.000004\n",
            "Epoch: 0711 loss = 0.000005\n",
            "Epoch: 0712 loss = 0.000005\n",
            "Epoch: 0713 loss = 0.000005\n",
            "Epoch: 0714 loss = 0.000003\n",
            "Epoch: 0715 loss = 0.000004\n",
            "Epoch: 0716 loss = 0.000008\n",
            "Epoch: 0717 loss = 0.000005\n",
            "Epoch: 0718 loss = 0.000004\n",
            "Epoch: 0719 loss = 0.000003\n",
            "Epoch: 0720 loss = 0.000009\n",
            "Epoch: 0721 loss = 0.000005\n",
            "Epoch: 0722 loss = 0.000004\n",
            "Epoch: 0723 loss = 0.000006\n",
            "Epoch: 0724 loss = 0.000004\n",
            "Epoch: 0725 loss = 0.000008\n",
            "Epoch: 0726 loss = 0.000004\n",
            "Epoch: 0727 loss = 0.000004\n",
            "Epoch: 0728 loss = 0.000003\n",
            "Epoch: 0729 loss = 0.000005\n",
            "Epoch: 0730 loss = 0.000006\n",
            "Epoch: 0731 loss = 0.000006\n",
            "Epoch: 0732 loss = 0.000005\n",
            "Epoch: 0733 loss = 0.000005\n",
            "Epoch: 0734 loss = 0.000007\n",
            "Epoch: 0735 loss = 0.000005\n",
            "Epoch: 0736 loss = 0.000005\n",
            "Epoch: 0737 loss = 0.000005\n",
            "Epoch: 0738 loss = 0.000005\n",
            "Epoch: 0739 loss = 0.000004\n",
            "Epoch: 0740 loss = 0.000007\n",
            "Epoch: 0741 loss = 0.000005\n",
            "Epoch: 0742 loss = 0.000004\n",
            "Epoch: 0743 loss = 0.000004\n",
            "Epoch: 0744 loss = 0.000007\n",
            "Epoch: 0745 loss = 0.000005\n",
            "Epoch: 0746 loss = 0.000005\n",
            "Epoch: 0747 loss = 0.000004\n",
            "Epoch: 0748 loss = 0.000006\n",
            "Epoch: 0749 loss = 0.000006\n",
            "Epoch: 0750 loss = 0.000004\n",
            "Epoch: 0751 loss = 0.000004\n",
            "Epoch: 0752 loss = 0.000006\n",
            "Epoch: 0753 loss = 0.000005\n",
            "Epoch: 0754 loss = 0.000004\n",
            "Epoch: 0755 loss = 0.000003\n",
            "Epoch: 0756 loss = 0.000009\n",
            "Epoch: 0757 loss = 0.000005\n",
            "Epoch: 0758 loss = 0.000006\n",
            "Epoch: 0759 loss = 0.000005\n",
            "Epoch: 0760 loss = 0.000007\n",
            "Epoch: 0761 loss = 0.000005\n",
            "Epoch: 0762 loss = 0.000005\n",
            "Epoch: 0763 loss = 0.000003\n",
            "Epoch: 0764 loss = 0.000004\n",
            "Epoch: 0765 loss = 0.000005\n",
            "Epoch: 0766 loss = 0.000005\n",
            "Epoch: 0767 loss = 0.000008\n",
            "Epoch: 0768 loss = 0.000006\n",
            "Epoch: 0769 loss = 0.000004\n",
            "Epoch: 0770 loss = 0.000006\n",
            "Epoch: 0771 loss = 0.000007\n",
            "Epoch: 0772 loss = 0.000005\n",
            "Epoch: 0773 loss = 0.000002\n",
            "Epoch: 0774 loss = 0.000005\n",
            "Epoch: 0775 loss = 0.000006\n",
            "Epoch: 0776 loss = 0.000003\n",
            "Epoch: 0777 loss = 0.000009\n",
            "Epoch: 0778 loss = 0.000005\n",
            "Epoch: 0779 loss = 0.000007\n",
            "Epoch: 0780 loss = 0.000004\n",
            "Epoch: 0781 loss = 0.000003\n",
            "Epoch: 0782 loss = 0.000006\n",
            "Epoch: 0783 loss = 0.000004\n",
            "Epoch: 0784 loss = 0.000009\n",
            "Epoch: 0785 loss = 0.000004\n",
            "Epoch: 0786 loss = 0.000004\n",
            "Epoch: 0787 loss = 0.000006\n",
            "Epoch: 0788 loss = 0.000006\n",
            "Epoch: 0789 loss = 0.000006\n",
            "Epoch: 0790 loss = 0.000004\n",
            "Epoch: 0791 loss = 0.000004\n",
            "Epoch: 0792 loss = 0.000003\n",
            "Epoch: 0793 loss = 0.000003\n",
            "Epoch: 0794 loss = 0.000005\n",
            "Epoch: 0795 loss = 0.000007\n",
            "Epoch: 0796 loss = 0.000005\n",
            "Epoch: 0797 loss = 0.000006\n",
            "Epoch: 0798 loss = 0.000017\n",
            "Epoch: 0799 loss = 0.000004\n",
            "Epoch: 0800 loss = 0.000005\n",
            "Epoch: 0801 loss = 0.000003\n",
            "Epoch: 0802 loss = 0.000005\n",
            "Epoch: 0803 loss = 0.000003\n",
            "Epoch: 0804 loss = 0.000006\n",
            "Epoch: 0805 loss = 0.000007\n",
            "Epoch: 0806 loss = 0.000006\n",
            "Epoch: 0807 loss = 0.000003\n",
            "Epoch: 0808 loss = 0.000005\n",
            "Epoch: 0809 loss = 0.000005\n",
            "Epoch: 0810 loss = 0.000004\n",
            "Epoch: 0811 loss = 0.000006\n",
            "Epoch: 0812 loss = 0.000005\n",
            "Epoch: 0813 loss = 0.000005\n",
            "Epoch: 0814 loss = 0.000007\n",
            "Epoch: 0815 loss = 0.000003\n",
            "Epoch: 0816 loss = 0.000004\n",
            "Epoch: 0817 loss = 0.000008\n",
            "Epoch: 0818 loss = 0.000005\n",
            "Epoch: 0819 loss = 0.000003\n",
            "Epoch: 0820 loss = 0.000005\n",
            "Epoch: 0821 loss = 0.000002\n",
            "Epoch: 0822 loss = 0.000005\n",
            "Epoch: 0823 loss = 0.000004\n",
            "Epoch: 0824 loss = 0.000006\n",
            "Epoch: 0825 loss = 0.000003\n",
            "Epoch: 0826 loss = 0.000002\n",
            "Epoch: 0827 loss = 0.000004\n",
            "Epoch: 0828 loss = 0.000004\n",
            "Epoch: 0829 loss = 0.000004\n",
            "Epoch: 0830 loss = 0.000004\n",
            "Epoch: 0831 loss = 0.000004\n",
            "Epoch: 0832 loss = 0.000004\n",
            "Epoch: 0833 loss = 0.000006\n",
            "Epoch: 0834 loss = 0.000005\n",
            "Epoch: 0835 loss = 0.000005\n",
            "Epoch: 0836 loss = 0.000013\n",
            "Epoch: 0837 loss = 0.000009\n",
            "Epoch: 0838 loss = 0.000005\n",
            "Epoch: 0839 loss = 0.000006\n",
            "Epoch: 0840 loss = 0.000003\n",
            "Epoch: 0841 loss = 0.000006\n",
            "Epoch: 0842 loss = 0.000003\n",
            "Epoch: 0843 loss = 0.000003\n",
            "Epoch: 0844 loss = 0.000005\n",
            "Epoch: 0845 loss = 0.000004\n",
            "Epoch: 0846 loss = 0.000004\n",
            "Epoch: 0847 loss = 0.000003\n",
            "Epoch: 0848 loss = 0.000003\n",
            "Epoch: 0849 loss = 0.000003\n",
            "Epoch: 0850 loss = 0.000006\n",
            "Epoch: 0851 loss = 0.000004\n",
            "Epoch: 0852 loss = 0.000005\n",
            "Epoch: 0853 loss = 0.000004\n",
            "Epoch: 0854 loss = 0.000005\n",
            "Epoch: 0855 loss = 0.000003\n",
            "Epoch: 0856 loss = 0.000003\n",
            "Epoch: 0857 loss = 0.000004\n",
            "Epoch: 0858 loss = 0.000004\n",
            "Epoch: 0859 loss = 0.000002\n",
            "Epoch: 0860 loss = 0.000004\n",
            "Epoch: 0861 loss = 0.000003\n",
            "Epoch: 0862 loss = 0.000003\n",
            "Epoch: 0863 loss = 0.000004\n",
            "Epoch: 0864 loss = 0.000004\n",
            "Epoch: 0865 loss = 0.000005\n",
            "Epoch: 0866 loss = 0.000003\n",
            "Epoch: 0867 loss = 0.000004\n",
            "Epoch: 0868 loss = 0.000005\n",
            "Epoch: 0869 loss = 0.000010\n",
            "Epoch: 0870 loss = 0.000005\n",
            "Epoch: 0871 loss = 0.000005\n",
            "Epoch: 0872 loss = 0.000004\n",
            "Epoch: 0873 loss = 0.000006\n",
            "Epoch: 0874 loss = 0.000005\n",
            "Epoch: 0875 loss = 0.000003\n",
            "Epoch: 0876 loss = 0.000004\n",
            "Epoch: 0877 loss = 0.000005\n",
            "Epoch: 0878 loss = 0.000004\n",
            "Epoch: 0879 loss = 0.000007\n",
            "Epoch: 0880 loss = 0.000003\n",
            "Epoch: 0881 loss = 0.000007\n",
            "Epoch: 0882 loss = 0.000003\n",
            "Epoch: 0883 loss = 0.000003\n",
            "Epoch: 0884 loss = 0.000004\n",
            "Epoch: 0885 loss = 0.000003\n",
            "Epoch: 0886 loss = 0.000004\n",
            "Epoch: 0887 loss = 0.000005\n",
            "Epoch: 0888 loss = 0.000004\n",
            "Epoch: 0889 loss = 0.000006\n",
            "Epoch: 0890 loss = 0.000005\n",
            "Epoch: 0891 loss = 0.000004\n",
            "Epoch: 0892 loss = 0.000004\n",
            "Epoch: 0893 loss = 0.000005\n",
            "Epoch: 0894 loss = 0.000006\n",
            "Epoch: 0895 loss = 0.000004\n",
            "Epoch: 0896 loss = 0.000004\n",
            "Epoch: 0897 loss = 0.000003\n",
            "Epoch: 0898 loss = 0.000004\n",
            "Epoch: 0899 loss = 0.000004\n",
            "Epoch: 0900 loss = 0.000003\n",
            "Epoch: 0901 loss = 0.000003\n",
            "Epoch: 0902 loss = 0.000004\n",
            "Epoch: 0903 loss = 0.000005\n",
            "Epoch: 0904 loss = 0.000003\n",
            "Epoch: 0905 loss = 0.000005\n",
            "Epoch: 0906 loss = 0.000003\n",
            "Epoch: 0907 loss = 0.000003\n",
            "Epoch: 0908 loss = 0.000003\n",
            "Epoch: 0909 loss = 0.000005\n",
            "Epoch: 0910 loss = 0.000004\n",
            "Epoch: 0911 loss = 0.000004\n",
            "Epoch: 0912 loss = 0.000003\n",
            "Epoch: 0913 loss = 0.000004\n",
            "Epoch: 0914 loss = 0.000005\n",
            "Epoch: 0915 loss = 0.000004\n",
            "Epoch: 0916 loss = 0.000005\n",
            "Epoch: 0917 loss = 0.000006\n",
            "Epoch: 0918 loss = 0.000004\n",
            "Epoch: 0919 loss = 0.000007\n",
            "Epoch: 0920 loss = 0.000004\n",
            "Epoch: 0921 loss = 0.000006\n",
            "Epoch: 0922 loss = 0.000003\n",
            "Epoch: 0923 loss = 0.000002\n",
            "Epoch: 0924 loss = 0.000004\n",
            "Epoch: 0925 loss = 0.000005\n",
            "Epoch: 0926 loss = 0.000004\n",
            "Epoch: 0927 loss = 0.000006\n",
            "Epoch: 0928 loss = 0.000008\n",
            "Epoch: 0929 loss = 0.000003\n",
            "Epoch: 0930 loss = 0.000003\n",
            "Epoch: 0931 loss = 0.000003\n",
            "Epoch: 0932 loss = 0.000004\n",
            "Epoch: 0933 loss = 0.000008\n",
            "Epoch: 0934 loss = 0.000005\n",
            "Epoch: 0935 loss = 0.000004\n",
            "Epoch: 0936 loss = 0.000005\n",
            "Epoch: 0937 loss = 0.000005\n",
            "Epoch: 0938 loss = 0.000004\n",
            "Epoch: 0939 loss = 0.000003\n",
            "Epoch: 0940 loss = 0.000006\n",
            "Epoch: 0941 loss = 0.000004\n",
            "Epoch: 0942 loss = 0.000003\n",
            "Epoch: 0943 loss = 0.000006\n",
            "Epoch: 0944 loss = 0.000004\n",
            "Epoch: 0945 loss = 0.000002\n",
            "Epoch: 0946 loss = 0.000007\n",
            "Epoch: 0947 loss = 0.000004\n",
            "Epoch: 0948 loss = 0.000005\n",
            "Epoch: 0949 loss = 0.000003\n",
            "Epoch: 0950 loss = 0.000006\n",
            "Epoch: 0951 loss = 0.000003\n",
            "Epoch: 0952 loss = 0.000003\n",
            "Epoch: 0953 loss = 0.000004\n",
            "Epoch: 0954 loss = 0.000004\n",
            "Epoch: 0955 loss = 0.000003\n",
            "Epoch: 0956 loss = 0.000004\n",
            "Epoch: 0957 loss = 0.000008\n",
            "Epoch: 0958 loss = 0.000004\n",
            "Epoch: 0959 loss = 0.000005\n",
            "Epoch: 0960 loss = 0.000004\n",
            "Epoch: 0961 loss = 0.000006\n",
            "Epoch: 0962 loss = 0.000003\n",
            "Epoch: 0963 loss = 0.000003\n",
            "Epoch: 0964 loss = 0.000003\n",
            "Epoch: 0965 loss = 0.000005\n",
            "Epoch: 0966 loss = 0.000005\n",
            "Epoch: 0967 loss = 0.000003\n",
            "Epoch: 0968 loss = 0.000002\n",
            "Epoch: 0969 loss = 0.000003\n",
            "Epoch: 0970 loss = 0.000005\n",
            "Epoch: 0971 loss = 0.000004\n",
            "Epoch: 0972 loss = 0.000004\n",
            "Epoch: 0973 loss = 0.000004\n",
            "Epoch: 0974 loss = 0.000002\n",
            "Epoch: 0975 loss = 0.000004\n",
            "Epoch: 0976 loss = 0.000003\n",
            "Epoch: 0977 loss = 0.000004\n",
            "Epoch: 0978 loss = 0.000005\n",
            "Epoch: 0979 loss = 0.000004\n",
            "Epoch: 0980 loss = 0.000006\n",
            "Epoch: 0981 loss = 0.000003\n",
            "Epoch: 0982 loss = 0.000003\n",
            "Epoch: 0983 loss = 0.000004\n",
            "Epoch: 0984 loss = 0.000004\n",
            "Epoch: 0985 loss = 0.000003\n",
            "Epoch: 0986 loss = 0.000004\n",
            "Epoch: 0987 loss = 0.000002\n",
            "Epoch: 0988 loss = 0.000004\n",
            "Epoch: 0989 loss = 0.000003\n",
            "Epoch: 0990 loss = 0.000004\n",
            "Epoch: 0991 loss = 0.000004\n",
            "Epoch: 0992 loss = 0.000002\n",
            "Epoch: 0993 loss = 0.000003\n",
            "Epoch: 0994 loss = 0.000005\n",
            "Epoch: 0995 loss = 0.000003\n",
            "Epoch: 0996 loss = 0.000004\n",
            "Epoch: 0997 loss = 0.000002\n",
            "Epoch: 0998 loss = 0.000003\n",
            "Epoch: 0999 loss = 0.000002\n",
            "Epoch: 1000 loss = 0.000003\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dGLlWp3gCl1X"
      },
      "source": [
        "def greedy_decoder(model, enc_input, start_symbol):\n",
        "    \"\"\"\n",
        "    For simplicity, a Greedy Decoder is Beam search when K=1. This is necessary for inference as we don't know the\n",
        "    target sequence input. Therefore we try to generate the target input word by word, then feed it into the transformer.\n",
        "    Starting Reference: http://nlp.seas.harvard.edu/2018/04/03/attention.html#greedy-decoding\n",
        "    :param model: Transformer Model\n",
        "    :param enc_input: The encoder input\n",
        "    :param start_symbol: The start symbol. In this example it is 'S' which corresponds to index 4\n",
        "    :return: The target input\n",
        "    \"\"\"\n",
        "    enc_outputs, enc_self_attns = model.encoder(enc_input)\n",
        "    dec_input = torch.zeros(1, 0).type_as(enc_input.data)\n",
        "    terminal = False\n",
        "    next_symbol = start_symbol\n",
        "    while not terminal:         \n",
        "        dec_input = torch.cat([dec_input.detach(),torch.tensor([[next_symbol]],dtype=enc_input.dtype).cuda()],-1)\n",
        "        dec_outputs, _, _ = model.decoder(dec_input, enc_input, enc_outputs)\n",
        "        projected = model.projection(dec_outputs)\n",
        "        prob = projected.squeeze(0).max(dim=-1, keepdim=False)[1]\n",
        "        next_word = prob.data[-1]\n",
        "        next_symbol = next_word\n",
        "        if next_symbol == tgt_vocab[\".\"]:\n",
        "            terminal = True\n",
        "        print(next_word)            \n",
        "    return dec_input"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EFsf8gHrgAl1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "888a265b-c77e-4fb5-af7a-9eadc0a52c7f"
      },
      "source": [
        "# Test\n",
        "enc_inputs, _, _ = next(iter(loader))\n",
        "enc_inputs = enc_inputs.cuda()\n",
        "for i in range(len(enc_inputs)):\n",
        "    greedy_dec_input = greedy_decoder(model, enc_inputs[i].view(1, -1), start_symbol=tgt_vocab[\"S\"])\n",
        "    predict, _, _, _ = model(enc_inputs[i].view(1, -1), greedy_dec_input)\n",
        "    predict = predict.data.max(1, keepdim=True)[1]\n",
        "    print(enc_inputs[i], '->', [idx2word[n.item()] for n in predict.squeeze()])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(1, device='cuda:0')\n",
            "tensor(2, device='cuda:0')\n",
            "tensor(3, device='cuda:0')\n",
            "tensor(4, device='cuda:0')\n",
            "tensor(8, device='cuda:0')\n",
            "tensor([1, 2, 3, 4, 0], device='cuda:0') -> ['i', 'want', 'a', 'beer', '.']\n",
            "tensor(1, device='cuda:0')\n",
            "tensor(2, device='cuda:0')\n",
            "tensor(3, device='cuda:0')\n",
            "tensor(5, device='cuda:0')\n",
            "tensor(8, device='cuda:0')\n",
            "tensor([1, 2, 3, 5, 0], device='cuda:0') -> ['i', 'want', 'a', 'coke', '.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Z2kTP5eQCPzB"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}